# -*- coding: utf-8 -*-
"""
Created on Sun Oct 21 22:51:53 2018

@author: HP
"""


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import matplotlib.pyplot as plt  # Matlab-style plotting
import seaborn as sns
color = sns.color_palette()
sns.set_style('darkgrid')
import warnings
def ignore_warn(*args, **kwargs):
    pass
warnings.warn = ignore_warn #ignore annoying warning (from sklearn and seaborn)
from scipy import stats
from scipy.stats import norm, skew #for some statistics

pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x)) #Limiting floats output to 3 decimal points

from subprocess import check_output

train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')

#Save the 'Id' column
train_ID = train['Id']
test_ID = test['Id']

#Now drop the  'Id' colum since it's unnecessary for  the prediction process.
train.drop("Id", axis = 1, inplace = True)
test.drop("Id", axis = 1, inplace = True)
#%%

train = train.drop(train[(train['GrLivArea']>4000) & (train['SalePrice']<300000)].index)
train["SalePrice"] = np.log1p(train["SalePrice"])
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat((train, test)).reset_index(drop=True)
all_data.drop(['SalePrice'], axis=1, inplace=True)
all_data_na = (all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})

all_data = all_data.drop(['Utilities','FireplaceQu','Fence','Alley','MiscFeature','PoolQC'],axis=1)

all_data["Functional"] = all_data["Functional"].fillna("Typ")
from my_fillna_model import my_fillna_model
li = ['Electrical','KitchenQual','Exterior1st','Exterior2nd','SaleType','LotFrontage',
      'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','GarageYrBlt', 'GarageArea', 'GarageCars',
      'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath',
      'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','MasVnrType',
      'MasVnrArea','MSZoning','MSSubClass']
all_data[li] = my_fillna_model(all_data[li])

#Check remaining missing values if any 
all_data_na = (all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
missing_data.head()
#MSSubClass=The building class
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
#Changing OverallCond into a categorical variable
all_data['OverallCond'] = all_data['OverallCond'].astype(str)
#Year and month sold are transformed into categorical features.
all_data['YrSold'] = all_data['YrSold'].astype(str)
all_data['MoSold'] = all_data['MoSold'].astype(str)
from sklearn.preprocessing import LabelEncoder
cols = ('BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 
        'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1', 
        'BsmtFinType2', 'Functional', 'BsmtExposure', 'GarageFinish', 'LandSlope',
        'LotShape', 'PavedDrive', 'Street', 'CentralAir', 'MSSubClass', 'OverallCond', 
        'YrSold', 'MoSold')
# process columns, apply LabelEncoder to categorical features
for c in cols:
    lbl = LabelEncoder() 
    lbl.fit(list(all_data[c].values)) 
    all_data[c] = lbl.transform(list(all_data[c].values))
# Adding total sqfootage feature 
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
# Check the skew of all numerical features
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
print("\nSkew in numerical features: \n")
skewness = pd.DataFrame({'Skew' :skewed_feats})
skewness.head(10)
skewness = skewness[abs(skewness) > 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
from scipy.special import boxcox1p
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
    #all_data[feat] += 1
    all_data[feat] = boxcox1p(all_data[feat], lam)
all_data = pd.get_dummies(all_data)
train = all_data[:ntrain]
test = all_data[ntrain:]


#%%
from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC, Ridge
from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
#import xgboost as xgb
#import lightgbm as lgb
from xgboost.sklearn import XGBRegressor
from lightgbm.sklearn import LGBMRegressor
n_folds = 5

def rmsle_cv(model):
    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(train.values)
    rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
    return(rmse)
#%%
lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=3))
ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
                                   max_depth=4, max_features='sqrt',
                                   min_samples_leaf=15, min_samples_split=10, 
                                   loss='huber', random_state =5)
#model_xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, 
#                         learning_rate=0.05, max_depth=3, 
#                         min_child_weight=1.7817, n_estimators=2200,
#                         reg_alpha=0.4640, reg_lambda=0.8571,
#                         subsample=0.5213, silent=1,
#                         random_state =7, nthread = -1)

model_xgb = XGBRegressor(learning_rate=0.08,reg_lambda=0.95,subsample=0.9, min_child_weight=2, max_depth=3,n_estimators=750, colsample_bytree=0.2)

model_lgb = LGBMRegressor(objective='regression',num_leaves=5,
                          learning_rate=0.05, n_estimators=720,
                          max_bin = 55, bagging_fraction = 0.8,
                          bagging_freq = 5, feature_fraction = 0.2319,
                          feature_fraction_seed=9, bagging_seed=9,
                          min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)



from sklearn.svm import SVR
svr = SVR( C=210, epsilon=0.02,gamma=0.000055)

from tpot.builtins import StackingEstimator
from sklearn.linear_model import LassoLarsCV

xgb_opt = make_pipeline(
    StackingEstimator(estimator=LassoLarsCV(normalize=False)),
    StackingEstimator(estimator=LassoLarsCV(normalize=False)),
    XGBRegressor(learning_rate=0.1, max_depth=8, min_child_weight=14, n_estimators=100, nthread=1, subsample=1.0)
)

ridge = make_pipeline(RobustScaler(), Ridge(alpha =0.01, random_state=3))

#%%

score = rmsle_cv(xgb_opt)
print("xgb_opt_by_tpot score: {:.4f} ({:.4f})".format(score.mean(), score.std()))


score = rmsle_cv(svr)
print("\nSVR score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))



score = rmsle_cv(ridge)
print("\nRidge score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))


score = rmsle_cv(lasso)
print("\nLasso score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
score = rmsle_cv(ENet)
print("ElasticNet score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
score = rmsle_cv(KRR)
print("Kernel Ridge score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
score = rmsle_cv(GBoost)
print("Gradient Boosting score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
score = rmsle_cv(model_xgb)
print("Xgboost score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
score = rmsle_cv(model_lgb)
print("LGBM score: {:.4f} ({:.4f})\n" .format(score.mean(), score.std()))

#%% ensemble
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, models):
        self.models = models
        
    # we define clones of the original models to fit the data in
    def fit(self, X, y):
        self.models_ = [clone(x) for x in self.models]
        
        # Train cloned base models
        for model in self.models_:
            model.fit(X, y)

        return self
    
    #Now we do the predictions for cloned models and average them
    def predict(self, X):
        predictions = np.column_stack([
            model.predict(X) for model in self.models_
        ])
        return np.mean(predictions, axis=1)  

averaged_models = AveragingModels(models = (ENet, KRR, lasso))

score = rmsle_cv(averaged_models)
print(" Averaged base models score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))

#%%
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, base_models, meta_model, n_folds=5):
        self.base_models = base_models
        self.meta_model = meta_model
        self.n_folds = n_folds
   
    # We again fit the data on clones of the original models
    def fit(self, X, y):
        self.base_models_ = [list() for x in self.base_models]
        self.meta_model_ = clone(self.meta_model)
        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
        
        # Train cloned base models then create out-of-fold predictions
        # that are needed to train the cloned meta-model
        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
        for i, model in enumerate(self.base_models):
            for train_index, holdout_index in kfold.split(X, y):
                instance = clone(model)
                self.base_models_[i].append(instance)
                instance.fit(X[train_index], y[train_index])
                y_pred = instance.predict(X[holdout_index])
                out_of_fold_predictions[holdout_index, i] = y_pred
                
        # Now train the cloned  meta-model using the out-of-fold predictions as new feature
        self.meta_model_.fit(out_of_fold_predictions, y)
        return self
   
    #Do the predictions of all base models on the test data and use the averaged predictions as 
    #meta-features for the final prediction which is done by the meta-model
    def predict(self, X):
        meta_features = np.column_stack([
            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)
            for base_models in self.base_models_ ])
        return self.meta_model_.predict(meta_features)
'''
Lasso score: 0.1112 (0.0073)
ElasticNet score: 0.1113 (0.0074)
Kernel Ridge score: 0.1146 (0.0080)
Gradient Boosting score: 0.1178 (0.0086)
Xgboost score: 0.1138 (0.0064)
LGBM score: 0.1150 (0.0055)
'''

#model_xgb = XGBRegressor(learning_rate=0.08,reg_lambda=0.95,subsample=0.9, min_child_weight=2, max_depth=3,n_estimators=750, colsample_bytree=0.2)
#model_xgb = XGBRegressor(learning_rate=0.08, max_depth=3,n_estimators=750, colsample_bytree=0.4)
model_xgb = XGBRegressor(learning_rate=0.08,subsample=0.85 ,reg_lambda=0.93, min_child_weight=4, max_depth=3,n_estimators=750, colsample_bytree=0.2)

model_lgb
GBoost

stacked_averaged_models = StackingAveragedModels(base_models = (lasso,svr,xgb_opt, ENet, KRR),
                                                 meta_model = lasso)

#%%
score = rmsle_cv(stacked_averaged_models)
print("Stacking Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))

# 0.1074 (0.0072) (averaged_models,lasso,model_xgb,model_lgb,GBoost, ENet, KRR)

###score: 0.1078  ENet, GBoost, KRR
###score: 0.1080 (0.0072)


#from mlxtend.classifier import StackingClassifier
#sclf = StackingClassifier(classifiers=[ENet, GBoost, KRR], 
#                          meta_classifier=lasso)

#score = rmsle_cv(stacked_averaged_models)
#print("Stacking Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))


#%%
def rmsle(y, y_pred):
    return np.sqrt(mean_squared_error(y, y_pred))
#%%
stacked_averaged_models.fit(train.values, y_train)
stacked_train_pred = stacked_averaged_models.predict(train.values)
# 
stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))

xgb_opt.fit(train, y_train)
xgb_opt_pred = np.expm1(xgb_opt.predict(test))

# lasso
lasso.fit(train, y_train)
lasso_pred = np.expm1(lasso.predict(test))
## XGBoost
#model_xgb.fit(train, y_train)
#xgb_train_pred = model_xgb.predict(train)
#xgb_pred = np.expm1(model_xgb.predict(test))
#print(rmsle(y_train, xgb_train_pred))
## LightGBM
#model_lgb.fit(train, y_train)
#lgb_train_pred = model_lgb.predict(train)
#lgb_pred = np.expm1(model_lgb.predict(test.values))
#print(rmsle(y_train, lgb_train_pred))

'''RMSE on the entire Train data when averaging'''

print('RMSLE score on train data:')
print(rmsle(y_train,stacked_train_pred*0.70 +
               xgb_train_pred*0.15 + lgb_train_pred*0.15 ))


svr.fit(train.values, y_train)
svr_pred = svr.predict(train.values)
# 
svr_pred = np.expm1(svr.predict(test.values))
print(rmsle(y_train, svr_pred))

#%%
ensemble = stacked_pred*0.70 + xgb_opt_pred*0.3
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('./res/submission.csv',index=False)


#%%
#Xgboost score: 0.1140 (0.0075)
model_xgb = XGBRegressor(learning_rate=0.08, min_child_weight=2, max_depth=3,n_estimators=750, colsample_bytree=0.2)
#Xgboost score: 0.1138 (0.0064)?
model_xgb = XGBRegressor(learning_rate=0.08,reg_lambda=0.95,subsample=0.9, min_child_weight=2, max_depth=3,n_estimators=750, colsample_bytree=0.2)
#Xgboost score: 0.1135 (0.0077)
model_xgb = XGBRegressor(learning_rate=0.08,reg_lambda=1,subsample=1, min_child_weight=3, max_depth=3,n_estimators=750, colsample_bytree=0.2)

xgb_model_opt = XGBRegressor(learning_rate=0.06, max_depth=3,n_estimators=500, colsample_bytree=0.2)

score = rmsle_cv(xgb_model_opt)
print("Xgboost score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))


#%%
training_X,validation_X, training_y,validation_y = train_test_split(train, y_train ,test_size=0.2, random_state=17)

xgb_params = {'max_depth':[3],
'n_estimators':[750],
'colsample_bytree':[0.2],
'min_child_weight':[3],
'subsample':[0.85, 0.9, 0.95],
'reg_lambda':[0.85,0.9,0.95],
'learning_rate':[0.08]}
xgb_params_df = ort.genSets(xgb_params,mode=2)

xgb_model = XGBRegressor(n_jobs=-1)
xgb = AdjustParam(training_X, training_y, 'neg_mean_squared_error')
xgb.run_ort(model=xgb_model, df_params=xgb_params_df)
xgb_ort_res = xgb.ort_res
xgb_param_res = xgb.param_res
################## 选参：选取极差和方差都小的
xgb_param_res

xgb_params = {'max_depth':[3],
'n_estimators':[750],
'colsample_bytree':[0.2],
'min_child_weight':[4],
'subsample':[0.85],
'reg_lambda':[0.93],
'learning_rate':[0.08],
'gamma':np.arange(0.003,0.006,0.0005)}

#0.1133 (0.0082)
model_xgb = XGBRegressor(subsample=0.85 ,reg_lambda=0.93, min_child_weight=4,learning_rate=0.08, max_depth=3,n_estimators=750, colsample_bytree=0.2)
#model_xgb = XGBRegressor(learning_rate=0.08,subsample=0.83 ,reg_lambda=0.93, min_child_weight=3.8, max_depth=3,n_estimators=750, colsample_bytree=0.2)

score = rmsle_cv(model_xgb)
print("Xgboost score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
model_xgb.fit(train.values, y_train)
model_xgb_pred = model_xgb.predict(train.values)
print(rmsle(y_train, model_xgb_pred))

#%% 
from sklearn.svm import SVR
svr = SVR( C=1.0, epsilon=0.1, cache_size=200)
score = rmsle_cv(svr)
print("\nSVR score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))

training_X,validation_X, training_y,validation_y = train_test_split(train, y_train ,test_size=0.2, random_state=17)

svr_params = {'C':[190,200,210],
'epsilon':[0.018,0.020,0.022],
'gamma':[0.000053,0.000055,0.000057]}
svr_params_df = ort.genSets(svr_params,mode=2)

svr_model = SVR()
svr = AdjustParam(training_X, training_y, 'neg_mean_squared_error')
svr.run_ort(model=svr_model, df_params=svr_params_df)
svr_ort_res = svr.ort_res
svr_param_res = svr.param_res
################## 选参：选取极差和方差都小的
svr_param_res

svr = SVR( C=210, epsilon=0.02,gamma=0.000055)
score = rmsle_cv(svr)
print("\nSVR score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))

#%%
from tpot import TPOTRegressor
from sklearn.model_selection import train_test_split
'''
参数
generation：遗传算法进化次数，可理解为迭代次数
population_size：每次进化中种群大小
num_cv_folds：交叉验证
scoring：也就是损失函数
'''
training_X,validation_X, training_y,validation_y = train_test_split(train, y_train ,test_size=0.2, random_state=17)

config_dict = {
        'xgboost.sklearn.XGBRegressor':{
        'learning_rate':[0.08],'subsample':[0.9],
        'reg_lambda':[0,95],'min_child_weight':[2],
        'max_depth':[3],'n_estimators':[750],
        'colsample_bytree':[0.2]
        },
        'sklearn.svm.SVR':dict(C=[210], epsilon=[0.02],gamma=[0.000055]),
        'sklearn.linear_model.ElasticNet':dict(alpha=[0.0005], l1_ratio=[.9], random_state=[3]),
        'sklearn.linear_model.Lasso':dict(alpha =[0.0005], random_state=[3]),
        'sklearn.linear_model.Ridge':dict(alpha =[0.01], random_state=[3]),
        'sklearn.kernel_ridge.KernelRidge':dict(alpha=[0.6], kernel=['polynomial'], degree=[2], coef0=[2.5]),
        'sklearn.ensemble.GradientBoostingRegressor':dict(n_estimators=[3000], learning_rate=[0.05],
                                   max_depth=[4], max_features=['sqrt'],
                                   min_samples_leaf=[15], min_samples_split=[10], 
                                   loss=['huber'], random_state =[5]),
        'lightgbm.sklearn.LGBMRegressor':dict(objective=['regression'],num_leaves=[5],
                          learning_rate=[0.05], n_estimators=[720],
                          max_bin = [55], bagging_fraction = [0.8],
                          bagging_freq = [5], feature_fraction = [0.2319],
                          feature_fraction_seed=[9], bagging_seed=[9],
                          min_data_in_leaf =[6], min_sum_hessian_in_leaf = [11])
}

tpot = TPOTRegressor(config_dict=config_dict,generations=100, population_size=15, verbosity=2,warm_start=True)
#train, y_train 
#training_X, training_y
tpot.fit(training_X, training_y)
print(tpot.score(validation_X, validation_y))
tpot.export('10tpot_price_predicet_pipeline.py')
tpot_predict = tpot.predict(validation_X)
rmsle(validation_y, tpot.predict(validation_X))

#tpot = TPOTRegressor(generations=50, population_size=15, verbosity=2)
'''
TPOTRegressor(config_dict=None, crossover_rate=0.1, cv=5,
       disable_update_check=False, early_stop=None, generations=50,
       max_eval_time_mins=5, max_time_mins=None, memory=None,
       mutation_rate=0.9, n_jobs=1, offspring_size=None,
       periodic_checkpoint_folder=None, population_size=15,
       random_state=None, scoring=None, subsample=1.0, use_dask=False,
       verbosity=2, warm_start=False)

config_dict = {
　　# Classifiers
　　'sklearn.naive_bayes.MultinomialNB': {
　　},
　　'sklearn.tree.DecisionTreeClassifier': {
　　'criterion': ["gini", "entropy"],
　　'max_depth': range(1, 11),
　　'min_samples_split': range(2, 21),
　　'min_samples_leaf': range(1, 21)
　　}
　　}
'''

tpot.fit(train, y_train)
print(tpot.score(train, y_train))
tpot.export('tpot_price_predicet_pipeline1.py')
tpot_predict = tpot.predict(train)
rmsle(y_train, tpot.predict(train))

#%% 生成的模型 做交叉验证
score = rmsle_cv(exported_pipeline)
print("exported_pipeline Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))


#%% 结果保存

tpot_predict_res = np.expm1(tpot.predict(test.values))
ensemble = tpot_predict_res
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('./res/submission.csv',index=False)

from sklearn.externals import joblib
joblib.dump(tpot, './tpot_house_price.pkl')
clf = joblib.load('./tpot.pkl')


#%% 选出来的模型
from sklearn.linear_model import ElasticNetCV
exported_pipeline = make_pipeline(
    RobustScaler(),
    ElasticNetCV(l1_ratio=0.9, tol=1e-05)
)
#0.1113
np.mean(rmsle_cv(exported_pipeline))


tpot_predict_res = np.expm1(tpot.predict(test.values))
ensemble = tpot_predict_res
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('./res/submission.csv',index=False)

#%%
from tpot.builtins import StackingEstimator
from sklearn.linear_model import LassoLarsCV
from sklearn.pipeline import make_pipeline

opt = make_pipeline(
    StackingEstimator(estimator=ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3)),
    StackingEstimator(estimator=XGBRegressor(colsample_bytree=0.2, learning_rate=0.08, max_depth=3, min_child_weight=2, n_estimators=750, reg_lambda=0, subsample=0.9)),
    KernelRidge(alpha=0.6, coef0=2.5, degree=2, kernel="polynomial")
)

score = rmsle_cv(opt)
print("Stacking Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))

opt.fit(train, y_train)

opt_predict_res = np.expm1(opt.predict(test.values))
ensemble = stacked_pred*0.1 + opt_predict_res*0.7 + lasso_pred*0.1+ xgb_opt_pred*0.1
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('./res/submission.csv',index=False)
